#! /usr/bin/env python
#coding=utf-8
#author linluxiang@gmail.com
#version 3.0.0

'''
Token是一个tuple，第一个位置表示类型，第二个表示实际的值，第三个表示的是行号。
词法分析的作用就是讲一个程序转化成为编译程序能够看懂的Token串。
'''

import symbols
import sys
import StringIO
ChangeState={}
lineNumber=1
tokenList=[]
currentTokens=[]
error=None
userIn=None
userOut=None
#-------------------定义直译函数-------------------
'''
preLex的作用是对输入的串进行初步的处理，将每个字符一一对应的转化成最简单的标记。
总共有6种最基本的标记。词法分析程序根据preLex分析的结果进行词法分析。
'''

def preLex(s):
    global currentTokens
    for eachLetter in s:
        if eachLetter in symbols.Letter:
            currentTokens.append(('Letter',eachLetter))
        elif eachLetter in symbols.SimpleOperator:
            currentTokens.append(('SimpleOperator',eachLetter))
        elif eachLetter in symbols.Blank:
            currentTokens.append(('Blank',eachLetter))
        elif eachLetter in symbols.Point:
            currentTokens.append(('Point',eachLetter))
        elif eachLetter in symbols.Underline:
            currentTokens.append(('Underline',eachLetter))
        elif eachLetter in symbols.Digit:
            currentTokens.append(('Digit',eachLetter))
        elif eachLetter in symbols.EOL:
            currentTokens.append(('EOL',eachLetter))
        elif eachLetter in symbols.Separation:
            currentTokens.append(('Separation',eachLetter))
        elif eachLetter in symbols.ArrayStart:
            currentTokens.append (('ArrayStart',eachLetter))
        elif eachLetter in symbols.ArrayEnd:
            currentTokens.append (('ArrayEnd',eachLetter))
     
#-------------------以下是数字处理模块------------------
'''
doneInteger函数的作用是当一个integer类型的state，遇到了使他变成done状态的condition而进行的操作。
函数将state加入了tokenList，表示已经识别出了一个token。同时，将token赋值为condition，分析condition
究竟是什么符号，是单字符号还是双字符号。
'''

def doneInteger(state,condition):
    global token,lineNumber
    tokenList.append((state[0],int(state[1]),lineNumber))
    if condition[0]=='EOL':
            lineNumber=lineNumber+1
    if condition[0]!='SimpleOperator':#如果是SimpleOperator，不能判断是否是单字符的操作符。所以要分开考虑。
        tokenList.append((condition[0],condition[1],lineNumber))
        token=()
    elif condition[0]=='SimpleOperator':
        token=(condition[0],condition[1])
#以下函数表明当一个Integer状态遇到各种类型的字符时，归结到done状态时的处理函数
ChangeState['Integer','Blank']=doneInteger#将状态转移函数绑定到状态上。
ChangeState['Integer','Separation']=doneInteger
ChangeState['Integer','EOL']=doneInteger
ChangeState['Integer','SimpleOperator']=doneInteger
ChangeState['Integer','ArrayEnd']=doneInteger
#ChangeState['Integer','EOF']=doneInteger

#遇到数字继续，整型内部变换
def changeIntegerToInteger(state,condition):
    global token,lineNumber
    token=(token[0],token[1]+condition[1])
ChangeState['Integer','Digit']=changeIntegerToInteger
#遇到小数点，把整型变成实型
def changeIntegerToReal(state,condition):
    global token,lineNumber
    token=('Real',token[1]+condition[1])
ChangeState['Integer','Point']=changeIntegerToReal
#实型内部变换
def changeRealToReal(state,condition):
    global token,lineNumber
    token=(token[0],token[1]+condition[1])
ChangeState['Real','Digit']=changeRealToReal
#终结实型
def doneReal(state,condition):
    global token,lineNumber
    tokenList.append((state[0],float(state[1]),lineNumber))
    if condition[0]=='EOL':
            lineNumber=lineNumber+1
    if condition[0]!='SimpleOperator':
        tokenList.append((condition[0],condition[1],lineNumber))
        token=()
    elif condition[0]=='SimpleOperator':
        token=(condition[0],condition[1])

#以下函数表明当一个Real状态遇到各种类型的字符时，归结到done状态时的处理函数
ChangeState['Real','Blank']=doneReal#将状态转移函数绑定到状态上。
ChangeState['Real','Separation']=doneReal
ChangeState['Real','EOL']=doneReal
ChangeState['Real','SimpleOperator']=doneReal
ChangeState['Real','ArrayEnd']=doneReal


#------------以下处理字符串--------------------------

#处理Identify的终结状态。根据字符串是否在keyword中判断属于何种symbol，所有串仅仅提供这一个终结态
def doneIdentify(state,condition):
    global token,lineNumber
    if state[1] in symbols.Keyword:
        tokenList.append(('Keyword',state[1],lineNumber))
    else:
        tokenList.append(('Identify',state[1],lineNumber))
    if condition[0]=='EOL':
            lineNumber=lineNumber+1
    if condition[0]!='SimpleOperator':
        tokenList.append((condition[0],condition[1],lineNumber))
        token=()
    elif condition[0]=='SimpleOperator':
        token=(condition[0],condition[1])
        
ChangeState['Identify','Blank']=doneIdentify
ChangeState['Identify','Separation']=doneIdentify
ChangeState['Identify','EOL']=doneIdentify
ChangeState['Identify','SimpleOperator']=doneIdentify
ChangeState['Identify','ArrayStart']=doneIdentify
ChangeState['Identify','ArrayEnd']=doneIdentify

#Identify内部变换，遇到数字和字母自己内部转移
def changeIdentifyToIdentify(state,condition):
    global token,lineNumber
    token=(token[0],token[1]+condition[1])
ChangeState['Identify','Digit']=changeIdentifyToIdentify
ChangeState['Identify','Letter']=changeIdentifyToIdentify

#遇到下划线变成另一状态，Underline
def changeIdentifyToUnderline(state,condition):
    global token,lineNumber
    token=('IdentifyWithUnderline',token[1]+condition[1])
ChangeState['Identify','Underline']=changeIdentifyToUnderline

#下划线状态内部变换
def changeUnderlineToUnderline(state,condition):
    global token,lineNumber
    token=('IdentifyWithUnderline',token[1]+condition[1])
ChangeState['IdentifyWithUnderline','Underline']=changeUnderlineToUnderline

#下划线状态转移成为标准Identify态
def changeUnderlineToIdentify(state,condition):
    global token,lineNumber
    token=('Identify',token[1]+condition[1])
ChangeState['IdentifyWithUnderline','Letter']=changeUnderlineToIdentify
ChangeState['IdentifyWithUnderline','Digit']=changeUnderlineToIdentify


#------------以下是处理Operator模块------------------

'''
Operator1是1个SimpleOperator组成的Symbol
'''
def doneOperator(state,condition):
    global token,lineNumber
    if state[1] in symbols.Assign:
        tokenList.append(('Assign',state[1],lineNumber))
    elif state[1] in symbols.Computing:
        tokenList.append(('Computing',state[1],lineNumber))
    elif state[1] in symbols.Comparation:
        tokenList.append(('Comparation',state[1],lineNumber))
    if condition[0] in('Blank','EOL','Separation'):
        tokenList.append((condition[0],condition[1],lineNumber))
        token=()
    elif condition[0]=='Digit':
        token=('Integer',condition[1])
    elif condition[0]=='Letter':
        token=('Identify',condition[1])
    if condition[0]=='EOL':
        lineNumber=lineNumber+1
ChangeState['SimpleOperator','Blank']=doneOperator
ChangeState['SimpleOperator','EOL']=doneOperator
ChangeState['SimpleOperator','Digit']=doneOperator
ChangeState['SimpleOperator','Separation']=doneOperator
ChangeState['SimpleOperator','Letter']=doneOperator
#ChangeState['SimpleOperator','Computing']=doneOperator

'''
Operator2是2个SimpleOperator组成的Symbol。
'''
def doneOperator2(state,condition):
    
    global token,lineNumber
    if state[1] in symbols.CommentStart:
        token=('CommentStart',state[1]+condition[1])
    elif state[1] in symbols.CommentEnd:
        tokenList.append(('CommentEnd',state[1],lineNumber))
        token=()#读到CommentEnd，自动将原来的token（类型是CommentStart）抛弃
    elif state[1] in symbols.CommentOneline:
        token=('CommentOneline',state[1],lineNumber)

    elif state[1] in symbols.Comparation:
        tokenList.append(('Comparation',state[1],lineNumber))
        if condition[0] in('Blank','EOL','Separation'):
            tokenList.append((condition[0],condition[1],lineNumber))
            token=()
        elif condition[0]=='Digit':
            token=('Integer',condition[1])
        elif condition[0]=='Letter':
            token=('Identify',condition[1])
    elif state[1] in symbols.Computing:
        tokenList.append(('Computing',state[1],lineNumber))
        if condition[0] in('Blank','EOL','Separation'):
            tokenList.append((condition[0],condition[1],lineNumber))
            token=()
        elif condition[0]=='Digit':
            token=('Integer',condition[1])
        elif condition[0]=='Letter':
            token=('Identify',condition[1])
    if condition[0]=='EOL':
            lineNumber=lineNumber+1
ChangeState['SimpleOperator2','Blank']=doneOperator2
ChangeState['SimpleOperator2','EOL']=doneOperator2
ChangeState['SimpleOperator2','Digit']=doneOperator2
ChangeState['SimpleOperator2','Separation']=doneOperator2
ChangeState['SimpleOperator2','Letter']=doneOperator2



def changeOperatorToOperator2(state,condition):
    global token,lineNumber
    #-------当接收到正负号的时候，添加一个token------
    if condition[1] in ('+','-'):
        if state[1] in symbols.Assign:
            tokenList.append(('Assign',state[1],lineNumber))
            tokenList.append(('Computing',condition[1],lineNumber))
            token=()
        elif state[1] in symbols.Computing:
            tokenList.append(('Computing',state[1],lineNumber))
            tokenList.append(('Computing',condition[1],lineNumber))
            token=()
        elif state[1] in symbols.Comparation:
            tokenList.append(('Comparation',state[1],lineNumber))
            tokenList.append(('Computing',condition[1],lineNumber))
            token=()
    elif condition[1]=='*':
        token=('CommentStart',token[1]+condition[1])
    
    elif condition[1]!='/':
        token=('SimpleOperator2',token[1]+condition[1])

    else:
        token=('CommentOneline',token[1]+condition[1])
ChangeState['SimpleOperator','SimpleOperator']=changeOperatorToOperator2

#-------------以下是处理注释---------------
def addtoComment(state,condition):
    global token,lineNumber
    if condition[1]=="*":
        token=(state[0],token[1]+condition[1])
    elif state[1][-1]=="*" and condition[1]=="/":
        token=()
    else:
        token=(state[0],token[1])
    if condition[0]=='EOL':
        lineNumber+=1
        
ChangeState['CommentStart','SimpleOperator']=addtoComment
ChangeState['CommentStart','Blank']=addtoComment
ChangeState['CommentStart','Digit']=addtoComment
ChangeState['CommentStart','Letter']=addtoComment
ChangeState['CommentStart','Separation']=addtoComment
ChangeState['CommentStart','EOL']=addtoComment
ChangeState['CommentStart','ArrayStart']=addtoComment
ChangeState['CommentStart','ArrayEnd']=addtoComment
ChangeState['CommentStart','Point']=addtoComment
ChangeState['CommentStart','Underline']=addtoComment

ChangeState['CommentOneline','Digit']=addtoComment
ChangeState['CommentOneline','Blank']=addtoComment
ChangeState['CommentOneline','Letter']=addtoComment
ChangeState['CommentOneline','Separation']=addtoComment
ChangeState['CommentOneline','SimpleOperator']=addtoComment
ChangeState['CommentOneline','ArrayStart']=addtoComment
ChangeState['CommentOneline','ArrayEnd']=addtoComment

def doneComment(state,condition):
    global token,lineNumber
    tokenList.append((state[0],token[1],lineNumber))
    if (state[0],condition[0])==('CommentOneline','EOL'):#如果是单行注释，那么直接token为空，将EOL略掉
        lineNumber+=1
        token=()
    else:
        token=(condition[0],condition[1])

ChangeState['CommentOneline','EOL']=doneComment
#ChangeState['CommentStart','SimpleOperator']=doneComment
#------------以下是获得token模块----------------------
'''
getTokens函数目的是为了获得token列表，通过修改一个全局变量表来获得所有的token。
'''
token=()
def getTokens():
    global lineNumber,token,tokenList
    for eachSymbol in currentTokens:
        if token==():
            if eachSymbol[0]=='EOL':
                lineNumber=lineNumber+1
            elif eachSymbol[0]=='Digit':#如果以数字开头，默认是进行Integer的识别,直到遇到小数点转为Real
                token=('Integer',eachSymbol[1],lineNumber)
            elif eachSymbol[0]=='Letter':
                token=('Identify',eachSymbol[1],lineNumber)
            elif eachSymbol[0]=='SimpleOperator':
                token=('SimpleOperator',eachSymbol[1],lineNumber)
            elif eachSymbol[0]=='Separation':
                tokenList.append(('Separation',eachSymbol[1],lineNumber))
            elif eachSymbol[0]=='ArrayStart':
                tokenList.append(('ArrayStart',eachSymbol[1],lineNumber))
        else:
            try:
                tokenChanger=ChangeState[token[0],eachSymbol[0]]#获得转换函数
                
                if tokenChanger!=None:
                    tokenChanger(token,eachSymbol)#调用转换函数对状态和读入字符进行操作
                else:
                    Error('Error')
            except KeyError:
                tokenList=[]
                Error('Error in line %d'%lineNumber)

'''
以下是后续处理模块，去除空格，换行以及注释
'''
def afterLex():
    global tokenList
    tokenList=[eachToken for eachToken in tokenList if eachToken[0] not in ('Blank','EOL','CommentStart','CommentOneline','CommentEnd')]


def Error(str):
    global error
    print>>userOut, str

#Domain
def getTokenList(lines):
    global ChangeState,lineNumber,tokenList,currentTokens,error,userIn,userOut
    lineNumber=1
    tokenList=[]
    currentTokens=[]
    if __name__ == '__main__':
        userIn=sys.stdin
        userOut=sys.stdout
    for line in lines:
        preLex(line)

    currentTokens.append(('EOL','\n'))
    
    getTokens()
    if tokenList!=[]:
        afterLex()
    return tokenList
'''
以下是main函数模块，参数是源代码的文件名。如果没有参数，那么可以在程序运行的时候动态的输入源代码的文件名。
'''
         
def main(argv=None):
    global ChangeState,lineNumber,tokenList,currentTokens
    
    option='y'
    while option=='y':
        if argv==None:
            filename=raw_input('Please input the filename:')
        else:
            filename=args
        file=open(filename,'rU')
        tokenList=getTokenList(file.readlines())
        print tokenList
        for token in tokenList:
            print 'Token type is "%s",token value is "%s",line number is %d'%(token[0],token[1],token[2])
        file.close()
        option=''
        while option!='y' and option!='n':
            option=raw_input('Please input "y" to analyze another file,"n" to quit the program:')


def funcForTest(line):
    global tokenList,currentTokens,lineNumber
    lineNumber=1
    tokenList=[]
    currentTokens=[]
    preLex(line);
    currentTokens.append(('EOL','\n'));
    getTokens();
    afterLex();
    return tokenList;

if __name__=='__main__':
    length=len(sys.argv)
    if length>1:
        main(sys.argv[1])
    else:main()
    
    

